{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  MovieLens 1M Dataset  \n",
    "https://grouplens.org/datasets/movielens/    \n",
    "\n",
    "参考文档 https://blog.csdn.net/chengcheng1394/article/details/78820529"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Platform : win32 [win32/linux]\n",
      "Systerm  : 3.6.10 (default, Mar  5 2020, 10:17:47) [MSC v.1900 64 bit (AMD64)] \n",
      "numpy  Version: 1.16.0\n",
      "pandas Version: 1.0.4\n",
      "tensorflow Version: 1.12.0\n"
     ]
    }
   ],
   "source": [
    "import os \n",
    "import sys\n",
    "import pickle\n",
    "import numpy as np \n",
    "import pandas as pd\n",
    "import tensorflow as tf \n",
    "np.set_printoptions(precision=3)   # 设置 numpy 显示位数\n",
    "print('Platform : {} [win32/linux]'.format(sys.platform))  # 当前平台信息 \n",
    "print('Systerm  : {} '.format(sys.version))\n",
    "print('numpy  Version: {}'.format(np.__version__))\n",
    "print('pandas Version: {}'.format(pd.__version__))\n",
    "print('tensorflow Version: {}'.format(tf.__version__))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 下载数据  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "from urllib.request import urlretrieve\n",
    "from tqdm import tqdm\n",
    "import zipfile\n",
    "import hashlib\n",
    "\n",
    "\n",
    "# 解压 zip 文件\n",
    "def _unzip(save_path, _, database_name, data_path):\n",
    "    \"\"\"\n",
    "    Unzip wrapper with the same interface as _ungzip\n",
    "    :param save_path: The path of the gzip files\n",
    "    :param database_name: Name of database\n",
    "    :param data_path: Path to extract to\n",
    "    :param _: HACK - Used to have to same interface as _ungzip\n",
    "    \"\"\"\n",
    "    print('Extracting {}...'.format(database_name))\n",
    "    with zipfile.ZipFile(save_path) as zf:\n",
    "        zf.extractall(data_path)\n",
    "\n",
    "\n",
    "# 下载数据\n",
    "def download_extract(database_name, data_path):\n",
    "    \"\"\"\n",
    "    Download and extract database \n",
    "    :param database_name: Database name\n",
    "    \"\"\"\n",
    "    DATASET_ML1M = 'ml-1m'\n",
    "\n",
    "    if database_name == DATASET_ML1M:\n",
    "        url = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'\n",
    "        hash_code = 'c4d9eecfca2ab87c1945afe126590906'\n",
    "        extract_path = os.path.join(data_path, 'ml-1m')\n",
    "        save_path = os.path.join(data_path, 'ml-1m.zip')\n",
    "        extract_fn = _unzip\n",
    "\n",
    "    if os.path.exists(extract_path):\n",
    "        print('Found {} Data'.format(database_name))\n",
    "        return\n",
    "\n",
    "    if not os.path.exists(data_path):\n",
    "        os.makedirs(data_path)\n",
    "\n",
    "    if not os.path.exists(save_path):\n",
    "        with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Downloading {}'.format(database_name)) as pbar:\n",
    "            urlretrieve(\n",
    "                url,\n",
    "                save_path,\n",
    "                pbar.hook)\n",
    "\n",
    "    assert hashlib.md5(open(save_path, 'rb').read()).hexdigest() == hash_code, \\\n",
    "        '{} file is corrupted.  Remove the file and try again.'.format(save_path)\n",
    "\n",
    "    os.makedirs(extract_path)\n",
    "    try:\n",
    "        extract_fn(save_path, extract_path, database_name, data_path)\n",
    "    except Exception as err:\n",
    "        shutil.rmtree(extract_path)  # Remove extraction folder if there is an error\n",
    "        raise err\n",
    "\n",
    "    print('Done.')\n",
    "\n",
    "\n",
    "class DLProgress(tqdm):\n",
    "    \"\"\"\n",
    "    Handle Progress Bar while Downloading\n",
    "    \"\"\"\n",
    "    last_block = 0\n",
    "\n",
    "    def hook(self, block_num=1, block_size=1, total_size=None):\n",
    "        \"\"\"\n",
    "        A hook function that will be called once on establishment of the network connection and\n",
    "        once after each block read thereafter.\n",
    "        :param block_num: A count of blocks transferred so far\n",
    "        :param block_size: Block size in bytes\n",
    "        :param total_size: The total size of the file. This may be -1 on older FTP servers which do not return\n",
    "                            a file size in response to a retrieval request.\n",
    "        \"\"\"\n",
    "        self.total = total_size\n",
    "        self.update((block_num - self.last_block) * block_size)\n",
    "        self.last_block = block_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found ml-1m Data\n",
      "MoviesLen 1M Data Save path: D:\\Desktop\\MovieLens\\data\n"
     ]
    }
   ],
   "source": [
    "# 下载数据       \n",
    "data_dir = os.path.abspath(os.path.join('.', 'data'))\n",
    "download_extract('ml-1m', data_dir)\n",
    "print('MoviesLen 1M Data Save path: {}'.format(data_dir))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--- \n",
    "## 数据情况 \n",
    "本项目使用的是MovieLens 1M 数据集，包含6000个用户在近4000部电影上的1亿条评论。\n",
    "\n",
    "数据集分为三个文件：用户数据users.dat，电影数据movies.dat和评分数据ratings.dat。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用户数据\n",
    "分别有用户ID、性别、年龄、职业ID和邮编等字段。\n",
    "\n",
    "数据中的格式：UserID::Gender::Age::Occupation::Zip-code\n",
    "\n",
    "- Gender is denoted by a \"M\" for male and \"F\" for female\n",
    "- Age is chosen from the following ranges:\n",
    "\n",
    "\t*  1:  \"Under 18\"\n",
    "\t* 18:  \"18-24\"\n",
    "\t* 25:  \"25-34\"\n",
    "\t* 35:  \"35-44\"\n",
    "\t* 45:  \"45-49\"\n",
    "\t* 50:  \"50-55\"\n",
    "\t* 56:  \"56+\"\n",
    "\n",
    "- Occupation is chosen from the following choices:\n",
    "\n",
    "\t*  0:  \"other\" or not specified\n",
    "\t*  1:  \"academic/educator\"\n",
    "\t*  2:  \"artist\"\n",
    "\t*  3:  \"clerical/admin\"\n",
    "\t*  4:  \"college/grad student\"\n",
    "\t*  5:  \"customer service\"\n",
    "\t*  6:  \"doctor/health care\"\n",
    "\t*  7:  \"executive/managerial\"\n",
    "\t*  8:  \"farmer\"\n",
    "\t*  9:  \"homemaker\"\n",
    "\t* 10:  \"K-12 student\"\n",
    "\t* 11:  \"lawyer\"\n",
    "\t* 12:  \"programmer\"\n",
    "\t* 13:  \"retired\"\n",
    "\t* 14:  \"sales/marketing\"\n",
    "\t* 15:  \"scientist\"\n",
    "\t* 16:  \"self-employed\"\n",
    "\t* 17:  \"technician/engineer\"\n",
    "\t* 18:  \"tradesman/craftsman\"\n",
    "\t* 19:  \"unemployed\"\n",
    "\t* 20:  \"writer\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Path: D:\\Desktop\\MovieLens\\data\\ml-1m\\users.dat \n",
      "\n",
      "User Shape: (6040, 5) \n",
      "\n",
      "User Head:\n",
      "   UserID Gender  Age  OccupationID Zip-code\n",
      "0       1      F    1            10    48067\n",
      "1       2      M   56            16    70072\n",
      "2       3      M   25            15    55117\n",
      "3       4      M   45             7    02460\n",
      "4       5      M   25            20    55455 \n"
     ]
    }
   ],
   "source": [
    "users_path = os.path.abspath(os.path.join('.', 'data','ml-1m', 'users.dat'))\n",
    "\n",
    "users_title = ['UserID', 'Gender', 'Age', 'OccupationID', 'Zip-code']\n",
    "users = pd.read_table(users_path,  sep='::', header=None, names=users_title, engine = 'python')\n",
    "\n",
    "# 数据路径\n",
    "print('Data Path: {} \\n'.format(users_path))\n",
    "# 用户数量 \n",
    "print('User Shape: {} \\n'.format(users.shape))\n",
    "# 样例 \n",
    "print('User Head:\\n{} '.format(users.head()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 电影数据\n",
    "分别有电影ID、电影名和电影风格等字段。\n",
    "\n",
    "数据中的格式：MovieID::Title::Genres\n",
    "\n",
    "- Titles are identical to titles provided by the IMDB (including\n",
    "year of release)\n",
    "- Genres are pipe-separated and are selected from the following genres:\n",
    "\n",
    "\t* Action\n",
    "\t* Adventure\n",
    "\t* Animation\n",
    "\t* Children's\n",
    "\t* Comedy\n",
    "\t* Crime\n",
    "\t* Documentary\n",
    "\t* Drama\n",
    "\t* Fantasy\n",
    "\t* Film-Noir\n",
    "\t* Horror\n",
    "\t* Musical\n",
    "\t* Mystery\n",
    "\t* Romance\n",
    "\t* Sci-Fi\n",
    "\t* Thriller\n",
    "\t* War\n",
    "\t* Western\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Path: D:\\Desktop\\MovieLens\\data\\ml-1m\\movies.dat \n",
      "\n",
      "User Shape: (3883, 3) \n",
      "\n",
      "User Head:\n",
      "   MovieID                               Title                        Genres\n",
      "0        1                    Toy Story (1995)   Animation|Children's|Comedy\n",
      "1        2                      Jumanji (1995)  Adventure|Children's|Fantasy\n",
      "2        3             Grumpier Old Men (1995)                Comedy|Romance\n",
      "3        4            Waiting to Exhale (1995)                  Comedy|Drama\n",
      "4        5  Father of the Bride Part II (1995)                        Comedy \n"
     ]
    }
   ],
   "source": [
    "movies_path = os.path.abspath(os.path.join('.', 'data','ml-1m', 'movies.dat'))\n",
    "\n",
    "movies_title = ['MovieID', 'Title', 'Genres']\n",
    "movies = pd.read_table(movies_path, sep='::', header=None, names=movies_title, engine = 'python')\n",
    "\n",
    "# 数据路径\n",
    "print('Data Path: {} \\n'.format(movies_path))\n",
    "# 用户数量 \n",
    "print('User Shape: {} \\n'.format(movies.shape))\n",
    "# 样例 \n",
    "print('User Head:\\n{} '.format(movies.head()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 评分数据\n",
    "分别有用户ID、电影ID、评分和时间戳等字段。\n",
    "\n",
    "数据中的格式：UserID::MovieID::Rating::Timestamp\n",
    "\n",
    "- UserIDs range between 1 and 6040 \n",
    "- MovieIDs range between 1 and 3952\n",
    "- Ratings are made on a 5-star scale (whole-star ratings only)\n",
    "- Timestamp is represented in seconds since the epoch as returned by time(2)\n",
    "- Each user has at least 20 ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Path: D:\\Desktop\\MovieLens\\data\\ml-1m\\ratings.dat \n",
      "\n",
      "User Shape: (1000209, 4) \n",
      "\n",
      "User Head:\n",
      "   UserID  MovieID  Rating  timestamps\n",
      "0       1     1193       5   978300760\n",
      "1       1      661       3   978302109\n",
      "2       1      914       3   978301968\n",
      "3       1     3408       4   978300275\n",
      "4       1     2355       5   978824291 \n"
     ]
    }
   ],
   "source": [
    "ratings_path = os.path.abspath(os.path.join('.', 'data','ml-1m', 'ratings.dat'))\n",
    "\n",
    "ratings_title = ['UserID','MovieID', 'Rating', 'timestamps']\n",
    "ratings = pd.read_table(ratings_path, sep='::', header=None, names=ratings_title, engine = 'python')\n",
    "\n",
    "# 数据路径\n",
    "print('Data Path: {} \\n'.format(ratings_path))\n",
    "# 用户数量 \n",
    "print('User Shape: {} \\n'.format(ratings.shape))\n",
    "# 样例 \n",
    "print('User Head:\\n{} '.format(ratings.head()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings[\"Datetime\"]=pd.to_datetime(ratings[\"timestamps\"],unit='s')    # 时间戳转换\n",
    "ratings['release_year']=pd.to_datetime(ratings['Datetime'],format='%Y-%m-%d').dt.year  # 年份\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserID</th>\n",
       "      <th>MovieID</th>\n",
       "      <th>Rating</th>\n",
       "      <th>timestamps</th>\n",
       "      <th>Datetime</th>\n",
       "      <th>release_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "      <td>2000-12-31 22:12:40</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "      <td>2000-12-31 22:35:09</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "      <td>2000-12-31 22:32:48</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "      <td>2000-12-31 22:04:35</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "      <td>2001-01-06 23:38:11</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000204</th>\n",
       "      <td>6040</td>\n",
       "      <td>1091</td>\n",
       "      <td>1</td>\n",
       "      <td>956716541</td>\n",
       "      <td>2000-04-26 02:35:41</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000205</th>\n",
       "      <td>6040</td>\n",
       "      <td>1094</td>\n",
       "      <td>5</td>\n",
       "      <td>956704887</td>\n",
       "      <td>2000-04-25 23:21:27</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000206</th>\n",
       "      <td>6040</td>\n",
       "      <td>562</td>\n",
       "      <td>5</td>\n",
       "      <td>956704746</td>\n",
       "      <td>2000-04-25 23:19:06</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000207</th>\n",
       "      <td>6040</td>\n",
       "      <td>1096</td>\n",
       "      <td>4</td>\n",
       "      <td>956715648</td>\n",
       "      <td>2000-04-26 02:20:48</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000208</th>\n",
       "      <td>6040</td>\n",
       "      <td>1097</td>\n",
       "      <td>4</td>\n",
       "      <td>956715569</td>\n",
       "      <td>2000-04-26 02:19:29</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000209 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         UserID  MovieID  Rating  timestamps            Datetime  release_year\n",
       "0             1     1193       5   978300760 2000-12-31 22:12:40          2000\n",
       "1             1      661       3   978302109 2000-12-31 22:35:09          2000\n",
       "2             1      914       3   978301968 2000-12-31 22:32:48          2000\n",
       "3             1     3408       4   978300275 2000-12-31 22:04:35          2000\n",
       "4             1     2355       5   978824291 2001-01-06 23:38:11          2001\n",
       "...         ...      ...     ...         ...                 ...           ...\n",
       "1000204    6040     1091       1   956716541 2000-04-26 02:35:41          2000\n",
       "1000205    6040     1094       5   956704887 2000-04-25 23:21:27          2000\n",
       "1000206    6040      562       5   956704746 2000-04-25 23:19:06          2000\n",
       "1000207    6040     1096       4   956715648 2000-04-26 02:20:48          2000\n",
       "1000208    6040     1097       4   956715569 2000-04-26 02:19:29          2000\n",
       "\n",
       "[1000209 rows x 6 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re \n",
    "\n",
    "# 导入预处理数据\n",
    "def load_data(data_path, debug=False):\n",
    "    \"\"\"\n",
    "    Load Dataset from File\n",
    "    \"\"\"\n",
    "    # ###############################　　读取User数据\n",
    "    users_title = ['UserID', 'Gender', 'Age', 'JobID', 'Zip-code']\n",
    "    users = pd.read_table(os.path.join(data_path, 'users.dat'), sep='::', header=None, names=users_title, engine='python')\n",
    "    users = users.filter(regex='UserID|Gender|Age|JobID')  # 去除 Zip-code 列数据\n",
    "    users_orig = users.values\n",
    "\n",
    "    # 改变User数据中性别和年龄\n",
    "    gender_map = {'F': 0, 'M': 1}\n",
    "    users['Gender'] = users['Gender'].map(gender_map)\n",
    "\n",
    "    age_map = {val: ii for ii, val in enumerate(sorted(set(users['Age'])))}  # 年龄数据加排序\n",
    "    # age_map = {val: ii for ii, val in enumerate(set(users['Age']))}  # TODO 加个排序更好？\n",
    "    users['Age'] = users['Age'].map(age_map)\n",
    "\n",
    "    if debug:\n",
    "        print('{}{:^20s}{}'.format('-'*30, 'User Data', '-'*30))\n",
    "        print('Gender Map: {}\\n'.format(gender_map))\n",
    "        print('Age    Map: {}\\n'.format(age_map))\n",
    "        print('users Head: {}\\n'.format(users.head()))\n",
    "\n",
    "    # ###############################　　读取Movie数据集\n",
    "    movies_title = ['MovieID', 'Title', 'Genres']\n",
    "    movies = pd.read_table(os.path.join(data_path, 'movies.dat'), sep='::', header=None, names=movies_title, engine='python')\n",
    "    movies_orig = movies.values\n",
    "\n",
    "    # 电影类型转数字字典\n",
    "    genres_set = set()\n",
    "    for val in movies['Genres'].str.split('|'):\n",
    "        genres_set.update(val)\n",
    "\n",
    "    genres_set.add('<PAD>')\n",
    "    genres2int = {val: ii for ii, val in enumerate(sorted(genres_set))}  # 加上排序\n",
    "\n",
    "    # 将电影类型转成等长数字列表，长度是18\n",
    "    genres_map = {val: [genres2int[row] for row in val.split('|')] for ii, val in enumerate(set(movies['Genres']))}\n",
    "\n",
    "    for key in genres_map:\n",
    "        for cnt in range(max(genres2int.values()) - len(genres_map[key])):\n",
    "            genres_map[key].insert(len(genres_map[key]) + cnt, genres2int['<PAD>'])\n",
    "\n",
    "    movies['Genres'] = movies['Genres'].map(genres_map)\n",
    "\n",
    "    # 将Title中的年份去掉\n",
    "    pattern = re.compile(r'^(.*)\\((\\d+)\\)$')  # TODO 为啥 去掉年份\n",
    "\n",
    "    title_map = {val: pattern.match(val).group(1) for ii, val in enumerate(set(movies['Title']))}\n",
    "    movies['Title'] = movies['Title'].map(title_map)\n",
    "\n",
    "    # 电影Title转数字字典\n",
    "    title_set = set()\n",
    "    for val in movies['Title'].str.split():  # TODO 电影名称 拆分 成字符？\n",
    "        title_set.update(val)\n",
    "\n",
    "    title_set.add('<PAD>')\n",
    "    title_set = sorted(title_set)  # 加上排序\n",
    "    title2int = {val: ii for ii, val in enumerate(title_set)}\n",
    "\n",
    "    # 将电影Title转成等长数字列表，长度是15\n",
    "    title_count = 15\n",
    "    title_map = {val: [title2int[row] for row in val.split()] for ii, val in enumerate(set(movies['Title']))}\n",
    "\n",
    "    for key in title_map:\n",
    "        for cnt in range(title_count - len(title_map[key])):\n",
    "            title_map[key].insert(len(title_map[key]) + cnt, title2int['<PAD>'])\n",
    "\n",
    "    movies['Title'] = movies['Title'].map(title_map)\n",
    "\n",
    "    if debug:\n",
    "        print('{}{:^20s}{}'.format('-' * 30, 'Movies Data', '-' * 30))\n",
    "        print('movies.Title  len:【{}】 Head: \\n{}\\n'.format(len(movies['Title'][0]), movies['Title'].head()))\n",
    "        print('genres2int: {}\\n'.format(genres2int))\n",
    "        print('movies.Genres len:【{}】 Head: \\n{}\\n'.format(len(movies['Genres'][0]), movies['Genres'].head()))\n",
    "\n",
    "    # ###############################　　读取评分数据集\n",
    "    ratings_title = ['UserID', 'MovieID', 'ratings', 'timestamps']\n",
    "    ratings = pd.read_table(os.path.join(data_path, 'ratings.dat'), sep='::', header=None, names=ratings_title, engine='python')\n",
    "    ratings = ratings.filter(regex='UserID|MovieID|ratings')\n",
    "\n",
    "    # 合并三个表\n",
    "    data = pd.merge(pd.merge(ratings, users), movies)\n",
    "\n",
    "    # 将数据分成X和y两张表\n",
    "    target_fields = ['ratings']\n",
    "    features_pd, targets_pd = data.drop(target_fields, axis=1), data[target_fields]\n",
    "\n",
    "    features = features_pd.values\n",
    "    targets_values = targets_pd.values\n",
    "\n",
    "    if debug:\n",
    "        print('title_count    电影名称转换长度:{}'.format(title_count))\n",
    "        print('title_set      电影名称转换列表')\n",
    "        print('genres2int     电影类型转换字典')\n",
    "\n",
    "        print('features       转换后特征数据（User+Movie）  Shape:【{}】'.format(features.shape))\n",
    "        print('targets_values 评分Label  Shape:【{}】'.format(targets_values.shape))\n",
    "\n",
    "        print('ratings 评分 Shape:【{}】'.format(ratings.shape))\n",
    "        print('users   用户 Shape:【{}】'.format(users.shape))\n",
    "        print('movies  电影 Shape:【{}】'.format(movies.shape))\n",
    "\n",
    "        print('data  合并三个表后数据  Shape:【{}】'.format(data.shape))\n",
    "        print('movies_orig  电影原始数据 Shape:【{}】'.format(movies_orig.shape))\n",
    "        print('users_orig   用户原始数据 Shape:【{}】'.format(users_orig.shape))\n",
    "    return title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------     User Data      ------------------------------\n",
      "Gender Map: {'F': 0, 'M': 1}\n",
      "\n",
      "Age    Map: {1: 0, 18: 1, 25: 2, 35: 3, 45: 4, 50: 5, 56: 6}\n",
      "\n",
      "users Head:    UserID  Gender  Age  JobID\n",
      "0       1       0    0     10\n",
      "1       2       1    6     16\n",
      "2       3       1    2     15\n",
      "3       4       1    4      7\n",
      "4       5       1    2     20\n",
      "\n",
      "------------------------------    Movies Data     ------------------------------\n",
      "movies.Title  len:【15】 Head: \n",
      "0    [4537, 4273, 282, 282, 282, 282, 282, 282, 282...\n",
      "1    [2534, 282, 282, 282, 282, 282, 282, 282, 282,...\n",
      "2    [2094, 3278, 2968, 282, 282, 282, 282, 282, 28...\n",
      "3    [4746, 5175, 1675, 282, 282, 282, 282, 282, 28...\n",
      "4    [1738, 5123, 5172, 818, 3372, 2352, 282, 282, ...\n",
      "Name: Title, dtype: object\n",
      "\n",
      "genres2int: {'<PAD>': 0, 'Action': 1, 'Adventure': 2, 'Animation': 3, \"Children's\": 4, 'Comedy': 5, 'Crime': 6, 'Documentary': 7, 'Drama': 8, 'Fantasy': 9, 'Film-Noir': 10, 'Horror': 11, 'Musical': 12, 'Mystery': 13, 'Romance': 14, 'Sci-Fi': 15, 'Thriller': 16, 'War': 17, 'Western': 18}\n",
      "\n",
      "movies.Genres len:【18】 Head: \n",
      "0    [3, 4, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
      "1    [2, 4, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
      "2    [5, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...\n",
      "3    [5, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
      "4    [5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
      "Name: Genres, dtype: object\n",
      "\n",
      "title_count    电影名称转换长度:15\n",
      "title_set      电影名称转换列表\n",
      "genres2int     电影类型转换字典\n",
      "features       转换后特征数据（User+Movie）  Shape:【(1000209, 7)】\n",
      "targets_values 评分Label  Shape:【(1000209, 1)】\n",
      "ratings 评分 Shape:【(1000209, 3)】\n",
      "users   用户 Shape:【(6040, 4)】\n",
      "movies  电影 Shape:【(3883, 3)】\n",
      "data  合并三个表后数据  Shape:【(1000209, 8)】\n",
      "movies_orig  电影原始数据 Shape:【(3883, 3)】\n",
      "users_orig   用户原始数据 Shape:【(6040, 4)】\n"
     ]
    }
   ],
   "source": [
    "data_path = os.path.abspath(os.path.join('.', 'data', 'ml-1m'))\n",
    "\n",
    "# 数据预处理\n",
    "title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig = load_data(data_path, debug=True);\n",
    "\n",
    "# 预测处理数据保存\n",
    "pre_data_path = os.path.abspath(os.path.join('.', '_temp', 'preprocess.p'))\n",
    "os.makedirs(os.path.dirname(pre_data_path), exist_ok=True)\n",
    "pickle.dump((title_count, title_set, genres2int, features, targets_values, ratings, users, movies, data, movies_orig, users_orig), open(pre_data_path, 'wb'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PY36-tf12",
   "language": "python",
   "name": "py36_b"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
